{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "demographic-kenya",
   "metadata": {},
   "source": [
    "## GPT2 trained on a Dialogue corpus\n",
    "\n",
    "[Huggingface repo here](https://huggingface.co/microsoft/DialoGPT-large)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "contained-convenience",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "import torch\n",
    "from transformers import pipeline  \n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('microsoft/DialoGPT-large')\n",
    "model = AutoModelForCausalLM.from_pretrained('microsoft/DialoGPT-large')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "realistic-leone",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> User:Hey what are you up to?\n",
      "DialoGPT: Nothing much. You?\n",
      ">> User:Nothing\n",
      "DialoGPT: That's good.\n"
     ]
    }
   ],
   "source": [
    "# Let's chat for 2 turns\n",
    "for step in range(2):\n",
    "    # encode the new user input, add the eos_token and return a tensor in Pytorch\n",
    "    new_user_input_ids = tokenizer.encode(input(\">> User:\") + tokenizer.eos_token, return_tensors='pt')\n",
    "\n",
    "    # append the new user input tokens to the chat history\n",
    "    bot_input_ids = torch.cat([chat_history_ids, new_user_input_ids], dim=-1) if step > 0 else new_user_input_ids\n",
    "\n",
    "    # generated a response while limiting the total chat history to 1000 tokens, \n",
    "    chat_history_ids = model.generate(bot_input_ids, max_length=1000, pad_token_id=tokenizer.eos_token_id)\n",
    "\n",
    "    # pretty print last ouput tokens from bot\n",
    "    print(\"DialoGPT: {}\".format(tokenizer.decode(chat_history_ids[:, bot_input_ids.shape[-1]:][0], skip_special_tokens=True)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "experienced-creator",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hey what are you up to?<|endoftext|>Nothing much. You?<|endoftext|>Nothing<|endoftext|>'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(bot_input_ids[0])  # note the endoftext tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "auburn-minutes",
   "metadata": {},
   "source": [
    "## Turkish GPT2\n",
    "\n",
    "[Huggingface repo here](https://huggingface.co/redrussianarmy/gpt2-turkish-cased)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "wanted-consortium",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Merhaba. Ben Bilmem\n"
     ]
    }
   ],
   "source": [
    "turkish_tokenizer = AutoTokenizer.from_pretrained(\"redrussianarmy/gpt2-turkish-cased\")\n",
    "\n",
    "turkish_model = AutoModelForCausalLM.from_pretrained(\"redrussianarmy/gpt2-turkish-cased\")\n",
    "\n",
    "turkish_generator = pipeline(\n",
    "    'text-generation', model=turkish_model, tokenizer=turkish_tokenizer\n",
    ")\n",
    "\n",
    "print(turkish_generator('Merhaba. Ben', max_length=5)[0]['generated_text'])  # Hi. I wouldn't know"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "compound-painting",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "streaming-partition",
   "metadata": {},
   "source": [
    "## Python code completion\n",
    "\n",
    "[Huggingface repo here](https://huggingface.co/Sentdex/GPyT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "italian-delhi",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"Sentdex/GPyT\")\n",
    "model = AutoModelForCausalLM.from_pretrained(\"Sentdex/GPyT\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "authentic-prophet",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import pandas as pd\n",
      "import numpy as np\n",
      "\n",
      "df = pd.read_csv('data/data/data\n"
     ]
    }
   ],
   "source": [
    "input_code = \"\"\"import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df = pd\"\"\"  # I'd expect a read_csv here\n",
    "\n",
    "converted = input_code.replace(\"\\n\", \"<N>\")\n",
    "tokenized = tokenizer.encode(converted, return_tensors='pt')\n",
    "resp = model.generate(tokenized, beams=3, max_length=tokenized.shape[1] + 10)\n",
    "\n",
    "decoded = tokenizer.decode(resp[0])\n",
    "reformatted = decoded.replace(\"<N>\",\"\\n\")\n",
    "\n",
    "print(reformatted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e613ac37",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2e7b4e10",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Examples inspired by https://nlpcloud.io/effectively-using-gpt-j-gpt-neo-gpt-3-alternatives-few-shot-learning.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a870d1f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "https://huggingface.co/EleutherAI/gpt-neo-1.3B\n",
    "\n",
    "GPT-Neo 1.3B is a transformer model designed using EleutherAI's replication of the GPT-3 architecture. \n",
    "GPT-Neo refers to the class of models, while 1.3B represents the number of parameters of this particular \n",
    "pre-trained model.\n",
    "\n",
    "GPT-Neo 1.3B was trained on the Pile, a large scale curated dataset created by EleutherAI \n",
    "for the purpose of training this model. https://pile.eleuther.ai\n",
    "'''\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"EleutherAI/gpt-neo-1.3B\")\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\"EleutherAI/gpt-neo-1.3B\")\n",
    "\n",
    "gpt_neo = pipeline(\n",
    "    'text-generation', model=model, tokenizer=tokenizer\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e3093afd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I love goin to the beach.\n",
      "Correction: I love going to the beach.\n",
      "###\n",
      "Let me hav it!\n",
      "Correction: Let me have it!\n",
      "###\n",
      "It have too many drawbacks.\n",
      "Correction: It has too many drawbacks.\n",
      "###\n",
      "I do not wan to go\n",
      "Correction: I do not want to go.\n",
      "###\n"
     ]
    }
   ],
   "source": [
    "# spelling correction\n",
    "for result in gpt_neo(\"\"\"I love goin to the beach.\n",
    "Correction: I love going to the beach.\n",
    "###\n",
    "Let me hav it!\n",
    "Correction: Let me have it!\n",
    "###\n",
    "It have too many drawbacks.\n",
    "Correction: It has too many drawbacks.\n",
    "###\n",
    "I do not wan to go\n",
    "Correction:\"\"\",\n",
    "    max_length=75, early_stopping=True):\n",
    "    print(result['generated_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "239edaad",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I want to start coding tomorrow because it seems to be so fun!\n",
      "Intent: start coding\n",
      "###\n",
      "Show me the last pictures you have please.\n",
      "Intent: show pictures\n",
      "###\n",
      "Search all these files as fast as possible.\n",
      "Intent: search files\n",
      "###\n",
      "Can you please teach me Chinese next week?\n",
      "Intent: learn Chinese next week\n",
      "###\n",
      "Please show me some of your games.\n",
      "Intent\n"
     ]
    }
   ],
   "source": [
    "# intent detection\n",
    "for result in gpt_neo(\"\"\"I want to start coding tomorrow because it seems to be so fun!\n",
    "Intent: start coding\n",
    "###\n",
    "Show me the last pictures you have please.\n",
    "Intent: show pictures\n",
    "###\n",
    "Search all these files as fast as possible.\n",
    "Intent: search files\n",
    "###\n",
    "Can you please teach me Chinese next week?\n",
    "Intent:\"\"\",\n",
    "    max_length=90, early_stopping=True):\n",
    "    print(result['generated_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "41d67312",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "description: a red button that says stop\n",
      "code: <button style=color:white; background-color:red;>Stop</button>\n",
      "###\n",
      "description: a blue box that contains yellow circles with red borders\n",
      "code: <div style=background-color: blue; padding: 20px;><div style=background-color: yellow; border: 5px solid red; border-radius: 50%; padding: 20px; width: 100px; height: 100px;>\n",
      "###\n",
      "description: a Headline saying Welcome to AI\n",
      "code: <span style=font-size: large;><h1><b>AI</b></h1>\n",
      "###\n",
      "description: a Headline saying Welcome\n"
     ]
    }
   ],
   "source": [
    "for result in gpt_neo(\"\"\"description: a red button that says stop\n",
    "code: <button style=color:white; background-color:red;>Stop</button>\n",
    "###\n",
    "description: a blue box that contains yellow circles with red borders\n",
    "code: <div style=background-color: blue; padding: 20px;><div style=background-color: yellow; border: 5px solid red; border-radius: 50%; padding: 20px; width: 100px; height: 100px;>\n",
    "###\n",
    "description: a Headline saying Welcome to AI\n",
    "code:\"\"\",\n",
    "    max_length=150, early_stopping=True):\n",
    "    print(result['generated_text'])\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "198d9d23",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HTML code\n",
      "description: a red button that says stop\n",
      "code: <button style=color:white; background-color:red;>Stop</button>\n",
      "###\n",
      "description: a blue box that contains yellow circles with red borders\n",
      "code: <div style=background-color: blue; padding: 20px;><div style=background-color: yellow; border: 5px solid red; border-radius: 50%; padding: 20px; width: 100px; height: 100px;>\n",
      "###\n",
      "description: a Headline saying Welcome to AI\n",
      "code: <h2>Welcome to AI</h2>\n",
      "###\n",
      "description: a circle with text saying something else\n",
      "code: <circle id='\n"
     ]
    }
   ],
   "source": [
    "# I will tweak their example a litte bit to add a prompt. Some Sinan wisdom. The headline is much simpler code now :)\n",
    "for result in gpt_neo(\"\"\"HTML code\n",
    "description: a red button that says stop\n",
    "code: <button style=color:white; background-color:red;>Stop</button>\n",
    "###\n",
    "description: a blue box that contains yellow circles with red borders\n",
    "code: <div style=background-color: blue; padding: 20px;><div style=background-color: yellow; border: 5px solid red; border-radius: 50%; padding: 20px; width: 100px; height: 100px;>\n",
    "###\n",
    "description: a Headline saying Welcome to AI\n",
    "code:\"\"\",\n",
    "    max_length=150, early_stopping=True):\n",
    "    print(result['generated_text'])\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9921ae8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
